{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Time series prediction using RNNs with TensorFlow </h1>\n",
    "\n",
    "This notebook illustrates:\n",
    "<ol>\n",
    "<li> Creating a Recurrent Neural Network in TensorFlow\n",
    "<li> Creating a Custom Estimator in tf.estimator\n",
    "</ol>\n",
    "\n",
    "<p>\n",
    "\n",
    "<h3> Simulate some time-series data </h3>\n",
    "\n",
    "Essentially a set of sinusoids with random amplitudes and frequencies."
   ]
  },
  {
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
 "!sudo chown -R jupyter:jupyter /home/jupyter/training-data-analyst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow==1.15.3 in /opt/conda/lib/python3.7/site-packages (1.15.3)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.12.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.1.0)\n",
      "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.15.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (0.9.0)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (3.11.4)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (0.34.2)\n",
      "Requirement already satisfied: six>=1.10.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.14.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.18.2)\n",
      "Requirement already satisfied: keras-applications>=1.0.8 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.0.8)\n",
      "Requirement already satisfied: tensorflow-estimator==1.15.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.15.1)\n",
      "Requirement already satisfied: google-pasta>=0.1.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (0.2.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.1.0)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (1.27.2)\n",
      "Requirement already satisfied: gast==0.2.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (0.2.2)\n",
      "Requirement already satisfied: astor>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (0.8.1)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow==1.15.3) (3.2.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.3) (46.1.3)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.3) (3.2.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/lib/python3.7/site-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow==1.15.3) (1.0.0)\n",
      "Requirement already satisfied: h5py in /opt/conda/lib/python3.7/site-packages (from keras-applications>=1.0.8->tensorflow==1.15.3) (2.10.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow==1.15.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd5jc1X3o//dn+szW2d4lrbTqnUUI0QQGUWwDxiYGO3ZccU3ie+8vvzg3v8e+ce5z024SO3EhxMbYjgFjDJgiiukgEGqoN1YraXvv08v5/TEjacusdiWtpN3R5/U8++zMKd85s9J+9sz5niLGGJRSSqUvy8VugFJKqfNLA71SSqU5DfRKKZXmNNArpVSa00CvlFJpznaxG5BKQUGBmT179sVuhlJKzRjbt2/vMsYUpsqbloF+9uzZbNu27WI3QymlZgwROT5eng7dKKVUmtNAr5RSaU4DvVJKpTkN9EopleY00CulVJrTQK+UUmlOA71SSqU5DfRKKZXmNNArpVSam5YrY5VSCb89/Nuzrnv3/LunsCVqJtMevVJKpTkN9EopleYmDPQiUikir4nIARHZJyJ/nqKMiMi/iUidiOwWkdXD8m4RkUPJvG9P9RtQSil1epPp0UeB/2GMWQSsBb4hIotHlbkVqEl+3Qf8BEBErMCPkvmLgXtT1FVKKXUeTRjojTGtxpgdyceDwAGgfFSxO4BfmoTNQK6IlAJrgDpjTL0xJgw8miyrlFLqAjmjMXoRmQ2sAt4blVUONA573pRMGy891bXvE5FtIrKts7PzTJqllFLqNCYd6EUkE/gd8C1jzMDo7BRVzGnSxyYa84AxptYYU1tYmPKQFKWUUmdhUvPoRcROIsj/2hjzRIoiTUDlsOcVQAvgGCddKaXUBTKZWTcC/Aw4YIz5l3GKPQ18Njn7Zi3Qb4xpBbYCNSIyR0QcwD3JskoppS6QyfTorwI+A+wRkZ3JtP8JVAEYY+4HNgK3AXWAH/h8Mi8qIt8EXgSswIPGmH1T+g6UUkqd1oSB3hjzNqnH2oeXMcA3xsnbSOIPgVJKqYtAV8YqpVSa00CvlFJpTgO9UkqlOQ30SimV5jTQK6VUmtNAr5RSaU4DvVJKpTkN9EopleY00CulVJrTQK+UUmlOA71SSqU5DfRKKZXmNNArpVSa00CvlFJpTgO9UkqlOQ30SimV5iY8eEREHgQ+AnQYY5amyP8L4NPDrrcIKDTG9IjIMWAQiAFRY0ztVDVcKaXU5EymR/8QcMt4mcaYfzLGrDTGrAT+CnjDGNMzrMj1yXwN8kopdRFMGOiNMW8CPROVS7oXeOScWqSUUmpKTdkYvYh4SPT8fzcs2QAvich2Eblvgvr3icg2EdnW2dk5Vc1SSqlL3lTejP0osGnUsM1VxpjVwK3AN0Tk2vEqG2MeMMbUGmNqCwsLp7BZSil1aZvKQH8Po4ZtjDEtye8dwJPAmil8PaWUUpMwJYFeRHKA64DfD0vLEJGsE4+BDcDeqXg9pZRSkzeZ6ZWPAOuBAhFpAr4L2AGMMfcni30MeMkY4xtWtRh4UkROvM7DxpgXpq7pSimlJmPCQG+MuXcSZR4iMQ1zeFo9sOJsG6aUUmpq6MpYpZRKcxrolVIqzWmgV0qpNKeBXiml0tyEN2OVUhdW728eO/nY2759cnVu0a2k1Pi0R6+UUmlOA71SSqU5DfRKKZXmNNArpVSa00CvlFJpTgO9UkqlOQ30SimV5jTQK6VUmtNAr5RSaU4DvVJKpTkN9EopleYmDPQi8qCIdIhIymMARWS9iPSLyM7k13eG5d0iIodEpE5Evj2VDVdKKTU5k+nRPwTcMkGZt4wxK5Nf3wMQESvwI+BWYDFwr4gsPpfGKqWUOnMTBnpjzJtAz1lcew1QZ4ypN8aEgUeBO87iOkoppc7BVI3RXykiu0TkeRFZkkwrBxqHlWlKpqUkIveJyDYR2dbZ2TlFzVJKKTUV+9HvAGYZY4ZE5DbgKaAGkBRlzXgXMcY8ADwAUFtbO245pRTQfWTk82PhsWUGhk5/jdrPT1171LR2zj16Y8yAMWYo+XgjYBeRAhI9+MphRSuAlnN9PaWUUmfmnAO9iJSIiCQfr0lesxvYCtSIyBwRcQD3AE+f6+sppZQ6MxMO3YjII8B6oEBEmoDvAnYAY8z9wCeAr4lIFAgA9xhjDBAVkW8CLwJW4EFjzL7z8i6UUkqNa8JAb4y5d4L8HwI/HCdvI7Dx7JqmlFJqKujKWKWUSnMa6JVSKs1poFdKqTSngV4ppdKcBnqllEpzGuiVUirNaaBXSqk0p4FeKaXSnAZ6pZRKcxrolVIqzWmgV0qpNKeBXiml0pwGeqWUSnMa6JVSKs1poFdKqTSngV4ppdLchIFeRB4UkQ4R2TtO/qdFZHfy6x0RWTEs75iI7BGRnSKybSobrpRSanIm06N/CLjlNPlHgeuMMcuBvwUeGJV/vTFmpTGm9uyaqJRS6lxM5ijBN0Vk9mny3xn2dDNQce7NUkopNVWmeoz+i8Dzw54b4CUR2S4i952uoojcJyLbRGRbZ2fnFDdLqfQUxRDHXOxmqGluwh79ZInI9SQC/dXDkq8yxrSISBHwBxE5aIx5M1V9Y8wDJId9amtr9X+uUqdhMGz0dvFqbg9ZMRt3hrOY5fBe7GapaWpKevQishz4KXCHMab7RLoxpiX5vQN4ElgzFa+n1KXu3ax+Xvb2sMSfid0Iv+3dzVAsdLGbpaapcw70IlIFPAF8xhhzeFh6hohknXgMbABSztxRSk1egAjP5HVS4/fw+fYyvtBWTiAe4V1fw8VumpqmJhy6EZFHgPVAgYg0Ad8F7ADGmPuB7wD5wI9FBCCanGFTDDyZTLMBDxtjXjgP70GpS8o7jgaC1ji39xQiCKURJ0tcxWz3N3Fd5hwclikbkVVpYjKzbu6dIP9LwJdSpNcDK8bWUEqdrTiGd5wN1AQ8VIRdJ9Mv81SwN9jO4VAXS90lF7GFajrSlbFKzSBHrb30WAJcMZgzIr3KkUuWxcneQNtFapmazjTQKzWD7LK3YTcWlvoyR6SLCAtdhRwN9xA18YvUOjVdaaBXaoYwGA7YOqiJ5uM0Y391q535REycpnDfRWidms400Cs1Q3RafHRZ/SyOFqXMn+3wYkGoD/Vc4Jap6U4DvVIzxD5bBwCLI6kDvdNio8SeRUNEe/RqJA30Ss0Q++2dlMYy8Rr3uGUqHTm0hAeI6Ti9GkYDvVIzQJgYR609LIwWnrZcpT2XKHHaIoMXqGVqJtBAr9QM0GDtIyaGudG805Yrs2cDaKBXI2igV2oGqLf1IAbmTBDoc6wuXGKjLaqBXp2igV6pGeCItYfSeBaexO4j4xIRiu2ZtEWGLlDL1Eygm2IoNY1JMEzW5v18q6mdUJYL+9peIsWn3464xJbFjkAzMRPHKtqXU9qjV2raivv9FD61iYyDjdSVCnn9MQqf2oS9vfe09YrtWURMnAZdOKWSNNArNQ0ZY+h/8klsfT5evXMW/3KXlca7ryLmceJ95X2IjT99ssSeBcChoJ7UphI00Cs1DQX37SN0+DD96xazdU4Mb9xNhjubvmuXY+8bIqN+/F59oS0DC8LBYMcFbLGazjTQKzXNxEMhBl98EVtpKb6lczhu66MqmtitMjSrmFCxl8xD3RBPfeKmVSwU2jI4qD16laSBXqlpxr95M/GBAbJvu41Ba5geS4BZsdyT+UMr52LzRXC2+8a9RqEtk6O6541KmjDQi8iDItIhIimPAZSEfxOROhHZLSKrh+XdIiKHknnfnsqGK5WO4j4fvnffxTl/Po6qKhqsiRuqVcMCfXB2MXGHFc+x8W+2Ftg8tEQGCMQj573NavqbTI/+IeCW0+TfCtQkv+4DfgIgIlbgR8n8xcC9IrL4XBqrVLrr/c1jmECAjGuvBaDB2o/FCBWx7FOFrFYCFdm4WgZT3pT1BIdY23kMgONbfgRdH1yIpqtpbMJAb4x5EzjdZ8A7gF+ahM1AroiUAmuAOmNMvTEmDDyaLKuUSsFEIvQ89BCOOXNwVFYCcNzaR0k8E+eoJS/Bskws0TiOLv+I9Cx/Hx/a8wyrehMnTR2L+eG9n0Bbyg/k6hIxFWP05UDjsOdNybTx0lMSkftEZJuIbOvs1JtI6tIz+MorRDs68KxbBySmWDbY+kYM25wQKsrAWARX66kVsLZomKsOvgzAkYU3I8DRuddAdjns/C8I6Lz6S9VUrIyVFGnmNOkpGWMeAB4AqK2tHbecUhdL728eO+M63k/+0eSv/1+/xl5RgXPePACaTA8BiTIrOjbQG7uVUKEHV+sQAysTaUsbd5AZHOT1JbcSzMinLJDD0cgArP4TeOPv4dBzsPLTZ/we1Mw3FT36JqBy2PMKoOU06UqpUYKHDuPftg3vvfcilsSv5QHTDEBVLCd1ndJM7AMhrL4wmYF+5rYdoL54Pl05JQDMcXo5Gu6BjAKYfS00bYMh/bR8KZqKQP808Nnk7Ju1QL8xphXYCtSIyBwRcQD3JMsqpUbpffhhxOkk9+N3nUzbH2/BYayUxLNS1gmVJA4Id7b7WHFsCzGLjX2VJye9MduRx/FQL3FjoHo9iAWOvXk+34aapiYcuhGRR4D1QIGINAHfhcQWesaY+4GNwG1AHeAHPp/Mi4rIN4EXASvwoDFm33l4D0rNaLGBAfqffprsj3wYa+6pYZqD8RYqYzlYUo6CQjTbSdxhJaOrj7LeRnZXXUbIcer0qTnOPAImSntkkFJXNpStgqYtsOijYHWc9/elpo8JA70x5t4J8g3wjXHyNpL4Q6CUGkf/U09hAgG8n/rUybSoiVFn2lgXqxq/ogjhfDee7kHCVgd1pSNnL89xJna5PBrupdSRDZVroHkbtO+FstWprqjSlK6MVeoiMvE4vQ8/gnvlStxLlpxMrzcdhIlROc74/AmxAjvxAeFIdg0x68i96isdiU8HTSd2scyfB64caN4+tW9CTXsa6JW6iHzvvEv42DG8n/7UiPSD8VZg/BuxJ+QXDADQGiobk1dky8QhVhpPBHqxQOkK6DwE0dAUtF7NFBrolbqIeh9+GGt+Plk33zwi/aBpIQsX+XHPuHUtxCjPawUxWJsCY/NFKLfn0BjuP5VYvBTi0USwV5cMDfRKXSThpmaGXnuN3Ls/gcUx8ubooXgLCyxlyDg3YgGKacdhjxLLdeCpT70YqtKRc6pHD5A3F+xu6NCVspcSDfRKXSS9Dz8MFgveT35yRHrIRKg3HSyS0tPWL6MZn/EQyMvE3TwAZuw6w0pHLo2RfsyJPIsV8msS+9+kKK/SkwZ6pS6CuN9P3+OPk7XhJuylIwP6B6adGIaFlrHj7id48OGVPlooJ+J1YQ1EsXePHb6pdOQQiEfoiQ3LK6iBQC/0Hpuqt6OmOQ30Sl0E/b//PfGBAfI+89kxeYfiiQXkpwv0ZbQQN0IrpUS8ibnz7saBMeVOzLwZMXyTX5P4fuyts22+mmE00Ct1gZl4nJ5f/ReuJUtwr1o5Jv9gvIU8Mikg9YpYjKGEVrrJJ4yTSI4zscFZ89hAX5Eq0GcWgzMLjmqgv1RMxaZmSqkJ7HurediTbVjq64l/4S/Z//bY7Z92R9qoilVwpMdF30DFmPwCcxSXhNgZvJ6+2AIAvHntOA5HCBw9tcBqX0cmEeNCgB0tAarbMk/mLcmfB0ffTIzTy/g3fFV60B69UheYvPIEJtsLtdeNyfPHfbRZupgdHXdHb6rYRdTYaY1Vn6pXVEBGR9eYsnaxkUc27Yw6TDy/BobaoLvu7N+ImjE00Ct1IbU2IHu2wHUfAfvY/WaOhuowYpgTSx3oxUSpYB/NsWpinFoJ6ysqwO4PYB/yj6lTjJd2M+rsIB2nv6RooFfqApIXfoNxODHrb0+ZXx86DDBuj76ED3AQoDG6YES6vzAfAE/n2F59seSN7dFnFIAnH5p0O4RLgQZ6pS6U7nZ472W45jbI9qYsciT0AQWxXLJMRsr8KnYTwkP7qM3OAgWJ67m7e8fUKcZLPz6CJnwqUQTKaxObnKm0p4FeqQtEXkycUGU23D1umfrQYWZHx96ABbCZEKUcpJFlGKwj8mIuF+EMz7iBHhjbq6+oTWyFEOwfU0elFw30Sl0IA73w9vOw9ibIK0pdJNZPR7Rt3PH5Mg5gI0Ijy1PmB/K9qQO9jBPoyy8DDDTvmPz7UDOSBnqlLgD5w+MQjWJuuWfcMvWhD4Dxx+er2IWPXLpHnNB5SiDfi6u7Z8zWBsXkAYy9IVt+WeK7Dt+kvUkFehG5RUQOiUidiHw7Rf5fiMjO5NdeEYmJSF4y75iI7Enm6f8odcmJ9ffD689A7bVQknpYBuCD4AEES8pA7zRDFHEk0ZuX1L+2gYI8rNEYzv7BEekZ4iIDF52M2vjMnZuYfaM3ZNPehIFeRKzAj4BbgcXAvSIy4igbY8w/GWNWGmNWAn8FvGHMiO7D9cn82ilsu1IzQu/DDyNBP+bW0x7WxuHQfmY55uDGOSavgr1YiNPAinHrB/JP3JDtGZOXmGI5dliHilpo2qobnKW5yfTo1wB1xph6Y0wYeBS44zTl7wUemYrGKTXTxf1+en7xS8yyK6By7vjlTIwPggeZ71qUMr+S3fRRzIAUj3uNQN74M2+KxEvH6B49JIZv/F3Q1zDBO1Ez2WQCfTnQOOx5UzJtDBHxALcAvxuWbICXRGS7iNx3tg1Vaibq++1vifX1YW771GnLNYaPEzQB5jsXj8nzmB4KaBj3JuwJcaeDUFZm6kBPLp30ETfxkRllqxLfW3ed/o2oGW0ygT7VRhjjfc77KLBp1LDNVcaY1SSGfr4hItemfBGR+0Rkm4hs6+zsnESzlJreTDRK988exHP55TBvyWnLHgrtB2C+a2ygr2QPwISBHhLz6d1dY4duivASJUYPg6MyFifG/Nt2T3htNXNNJtA3wYjb/BXA2J2YEu5h1LCNMaYl+b0DeJLEUNAYxpgHjDG1xpjawsLCSTRLqektsHMn0Y4O8r/ylQnLHg7ux2vNo9A2dmimkt10UYVfUi+yGvGa+Xm4evsgPrLnfmKKZcfoKZYODxQsgFYN9OlsMoF+K1AjInNExEEimD89upCI5ADXAb8flpYhIlknHgMbAD3DTKU9E4vh27QJ19KlZFy1bsLyh4MHqHEtRkbtJJlt2silfVK9eUjckLXE4rj6Rm5ZXJRcNNVhUozTly7XHn2amzDQG2OiwDeBF4EDwGPGmH0i8lUR+eqwoh8DXjLG+IalFQNvi8guYAvwnDHmhalrvlLTU/DAAWI9PeR/+ctjgvdofdEeOqKtLHCOvRFbxW7iWGhi6aReN5CfmDM/evgmn2wsyNhFUwAly2GwFYZ0yDRdTWo/emPMRmDjqLT7Rz1/CHhoVFo9nGY+mFJpyBiD7+23sebnk3XjhyYsf3i88XljqGQ3HcwlJJkpao4VzMvFMHbmjU2sFJAzdugGEj16gLZdMO/GSb2Omll0ZaxSUyx89CjR1lYy1q1DrNYJyx8K7scudmY7R06/zIjVkUEfDZMctgGI222EcrJw94wN6IXk0pFqLn1J8vo68yZtaaBXaor53n4bS2Ym7hWT+zC7L7CLGuci7DJyf/r8yLvEsNFC6rn140lshZB6z5uUc+nduZA7S2/IpjEN9EpNoUhLC+EjR/CsXYvY7ROWH4oNcjxczxL3qD8KJkZeZAutLCAqrjNqQzDPi6u3H2KxEelFqbYrPkFvyKY1PTNWqSnk27QJcTrx1CZ2++j9TWJrYn/L2G0NAN6378dkGeYdd+E/cmorqOzofuxm4LRbHownkO/FEo8T7+2BglNTlU/MvEnseeMZWalkBRx4BoID4Mo+49dU05v26JWaIrH+foL79+O+7DIsbvek6hy01+MwduaM2sgsP/IuUTy0UXPG7Tix5w1dI2fRFEsukGK7YoDS5B+Udp39nI400Cs1RfxbE5uDedakXBOY0kFbPfMjs7EN+3AtJow3so1e+2XEZeLhn9GCeYmATmfHiPRTc+lPM/NGx+nTkgZ6paaAiUTwb9+Oc8ECbN6JV7AC9MkgzbYOFkWrR6TnRndiJUiP/cqzakvcbieUnYWM2kokEzdunKlvyGaVQEaRzrxJUxrolZoCwX37MH7/GfXmd9sPAbAsMnJ4piD8FmHxMmAbu+/NZAXyvdA1skcvIuNvVwzJG7J7zvo11fSlgV6pKeDfsgVrQQGO6uqJCyftdhzGG8umPHZqfxt7vI+c6G66HFePe8CIMRA3NmJxB8aMcwhJXi50d0F85MybQnJTL5qCxHz6zgMQDU36PaiZQWfdKHWOIi0tRJqbybrttgm3OzghSpR99jquDK1Ahm0Qmx/ZhGDosl89onwsaicQyCEcziQacWHMqYVYFgliOe7H7u3FmjWESKJHL7EYprcX8gtOli2SXHaaOuLGjO3llSyDeBQ6DkDZyjP+OajpSwO9UufIv2MH2Gy4l09+Beth23FCEmZZZP6pRGMoCL/NoLWGkLUUYyAUzMTnKyASzgDAZvfjcvdhjfUixIkbG9F4JqH+PKI9eVjcfpwVLQSHz7wZFuiL8RIhSlfUx5gjyk/MvGnbo4E+zWigV2qStrenOFs1GqN01/sE5xTz/sB+GBhbBCCHkTtY7nQcxGZsLI6c2vYgI1aPO97MUffn6fdZaepy4gtmYbGGychsx+3ux2qLJAqP2oDMV51HtDeXUGsJgQ/mEcvPJC5PJ2beLDi1srZIvGCgKdw/NtB754AjUxdOpSEN9EqdA3d9K5ZwFN+iqknXiRNnm2MvyyI1ODm17UFB5C36YmVs7txAz5ALhy1OVk4zbncfE40IicVgz+/F5u0j1FxGuKuEXav+jKVdbzN8t50TUyybIv2sHn0RiwWKl+oUyzSkgV6pc+A52EA020O4vGDiwkl1tgb6LINcHj619bDEQzT05rNt6PsYbFQUhCjxhmkcSjEV8jTEYnBVNmP1+Ok7Xs3OoIcVUcFmSxwKV0gOAjSFx7lu6XLY+XDi4BKLztVIF/ovqdRZsg74cDV14V9YyYRd7mG2OvZiNzZWhBcCEAhZONRg5b3BT+F1D7Fsjo+y/PA5xVl7fi9LI28x5Cxh974sYskDp+xiI49smsL9qSuWLIPwEPQePfsXV9OOBnqlzpLnYCMG8C+onLDsCTFibHPsZXlkAS7jpKXbwd7jHgIRO9d5H2JOuRWnfbwjmc9Mfl6EJQd+zsCgnYOHMzHJyxbhpSkyXqA/sTe9Dt+kk0kFehG5RUQOiUidiHw7Rf56EekXkZ3Jr+9Mtq5SM5IxeA42EqosJJblmbh80h77B/Rbhriify37Gzw0dTkp8PTxqfxvkpnjRSyT/2QwocJCijrfpzrjGJ1dTppaErtgFot3/B590SKw2HScPs1MOEYvIlbgR8BNJA4K3yoiTxtj9o8q+pYx5iNnWVepGcXR2oNtKMDAlWe2V/wb9u1c1XA7pnkZIathXlmAy6w/xRkJ0e2Y+GzZM5LcubLS9z79eWXUH/OQmxOh0JNLZ9RHMBrEZRu1BbLNCYULtUefZibTo18D1Blj6o0xYeBR4I5JXv9c6io1bbk/aCZusxKcXTLpOs0hP/MP3s6y5g/hzYqybLafoowe8iJb6HJcTfwM952fkNOFycrG0tXJwpoh7PY4+w9mURRL3DhuHmpOXa9Et0JIN5MJ9OVA47DnTcm00a4UkV0i8ryILDnDukrNHPE47iMtBGcXY+wTT1yLx60c73DSdKwIZ8xNSUUH88qC2G2GwvDrWIjS4Zj4bNmzUlgEXR3Y7YbFC4YIBC04jyZ+PY8PHE9dp2QZDLXDYPv5aZO64CYzvTLVoOHou0U7gFnGmCERuQ14CqiZZN3Ei4jcB9wHUFU1+TnJSl0w3UcAcLYNYQ2GCRRbT6aNxxeqoH1gPZG4g4NFmwjk7+SGzqsgCEKUYuvz9Ju5BPuCQIprhc5seuUYhYWw/TiYOLk5UaoqgjQ05VFWMI+GgYbUdUqH3ZDNuuncXl9NC5Pp0TcBw6cVVAAtwwsYYwaMMUPJxxsBu4gUTKbusGs8YIypNcbUFhYWpiqi1LTgbugnbrcQLM0ct0ws7qS1/3qa+m4HidFdsZE35j7Gbf2nxvQLZCd28dFqrjpvbTUFRUg0An2JPxizKv24XDHWH72HY73j9OiLk/P7dZw+bUwm0G8FakRkjog4gHuAp4cXEJESSe7mJCJrktftnkxdpWaUWBx30wDB8iywjv31MQYGg9Uc7b6XgeAC8jK2U1T4CBvLXmPlUCVV4bxkyTgllnfxmTIGzezz194TRwkmT5uyWmH+XB/ZgULC27NS19HDwtPOhEM3xpioiHwTeBGwAg8aY/aJyFeT+fcDnwC+JiJRIADcY4wxQMq65+m9KHXeudqGsETi+KtyxuRFYx7aB69lKFSN09ZBRe4zuOzdPJ7XTsAS4WPdpzYdyJN9uKSHutjdpB7hnCInPh13dkDNgsRreyP4S+soObSUvnY/ucUppofq3vRpZVJbICSHYzaOSrt/2OMfAj+cbF2lZip3wwBxh5VQ8alhG2OgP7CIzqF1GGOlMPMdvJ5diBiaHEHeye7jQ32LqEz25oUY5ZbX8Zsies2ZTc88Yy43Jitx2tTwm2M5y/fj66jgzccPcfs3Vo2tV7I8cVh4aBCc4/T81YyhK2OVmiSJxnE1DxCoyIbkwqZwNJum3ttpH7wep62L2fm/IS9jJyKGsMT5dVErmTErd/acCqb5sguX9NAUv4Hz2ps/oaBwzGlTs7Ld7Ch/icY9vTQd7Blb5+QKWT0sPB1ooFdqklwtg1hiBn9VDsZAj285x7o/STBaSHHWa1R6f4/DdmrF6VP5HbQ5wny6sxRP3AmAhQjlljcYMhX0m/njvdTUKiyCri4w8ZNJs51e9pS+gS3bsOl3dcTjoybDlSxLfNfhm7SggV6pSXI39BNz2RjKK6Wx9046h67G42hmdv6j5HoOjNjX7M3sXt7N7ueGvjwWBDJOppda3sIhAzTGbuSC9OZJzryJhKH/1B+hSkcuMUsUc0U7XY1DHNrcOrJSdhl48qFNDwtPBxrolZqE2FRAdOcAACAASURBVNAQzlYfRxdu4FjPJwlF8ynJfoXy3I3Yrb4RZbdm9vNUfgfLfJl8uOfU9sVOuimRd+iKL2eIWReu8aNm3gB4LHaKPEU0FO6leE42m39fTyQ07HxZkUSvXnv0aUEDvVKT0PbMq7y//M9pyP3wyV58jvvQmN2JX8vp4eGiNuYGPfxxRymWk712wyzLRgxWmuI3XtjGD595M8ycnDkcHTjKVZ+owd8fZterjSPrlSxPnB8bDV+ghqrzRQO9UhOo297Bs687GMospyQrdS8+juGpvA6ezu9k5VAWX2ktx2FO/XoVyjZyLPU0xm8kwgWexeL2JGbetI/c0mBe7jyO9B+huDqLOSsKeP/F4wSGhgX1spUQC0OHzoie6TTQKzWOaDjG6w8f4sX/3ItnqIUlvT8hxzO2Fx8lMbvmjdxerunP5TMdpdiG/Wp58FFpeYm++Dw6Te0FfhdJxaXQPnIcvjqnmkA0QJuvjbV3ziUSirH9+WGrZcuTbW1OcVaumlE00CuVQl+Hn8f/YRv73mxmUZWf1Tv+hXj52G2aghLjgZJmdmQO8pHuAj7WXTRsuAasRFnOLuLYORa/nQt1A3aMktLEzJtI5GTS3NzEweRH+o6QV5rBonWl7HmjiYGuQKJAbhV4CqBJA/1Mp2fGqrS1761xtuGdQHeLjwObWkCEZevLKXj0b6ColE7HavCfCtQBS5j/mv0Ore4AdzStZmXfLEYe5xFnnfNZPFY/h+OfufBDNsOY4hIsJo7p7ICyxAayc3NOBfprKq7h8o9Uc2hLO+89U89Nn1+SuCFbUQvN2y5au9XU0ECvzsnD742zA+I0EDo6zilK4zDGEG0JEDnuQzxWnAtzaOhpp+DgTpqv/jhDkVOzUvzWEI9Xb6bHNcQd9bXM7S9hiOjwq7HO8xpltqNs9l9Lb7gcOPObmkMmOnEhoGdo7LWPBk7dR7A5c5kFdNUdY9CRy9DRHo7EBsiw5vKHut04fIl/R/vCbA6/105rsQNbvpNPldfC4Rcg2A+usds+qJlBh26UAkzcEP5gkMhxH9Z8J65lXiwuK3kHNiMYuhedOv0pZInweM1mel1D3Hnkcub2jzx8RIiz1vMG850H2Bmo5WBo2YV+O2NEs3KIORw4R62QLXDOojN0alzeucyLOCwEtncnEsqT+/M077hQTVXngQZ6dckzkTihfX3EukLYqzw45mch1sQQTcH+TfgKqwgUVgAQI86zc7bT7R7kjiOXM3uwaMS1nBLgpsynWejcx57ganYG11zw95OSCOH8YpzdI2feFDgq6Q43YpInh1ucVlwrvESb/URa/FB+WaKg3pCd0TTQq0taPBgjuKeP+FAUx/xs7BUZJHfcxtnXQVbzYbqXXH2y/FvlBziW08mNDcvHBPl8azsfzX6MYlsbb/tuYHvgSi7azdcUQgVFOLo7IX5qK4QC5yzC8QAD0VOLqZwLc5AMG4Ht3RhXDuTXaKCf4TTQq0tWbDBCcE8vJhrHuSQXW4FzRH7+/k0AdC9ODNvU5bSxvbielR2zWdY98hS0Gsc+bst6AoywcfAu6sLneVfKsxAqLMUSjeDo7TqZVuScDUBH8OjJNLFZcK/KI9YV4siOTqhcA43vJbbpVDOS3oxVl6RYX5jQwX7EbsG5OAeLe+yvQsG+txmoWEAotwif6efFWTsp9uVwXfPik2WsRLnC8ybznQdojlTypm8DITP5Q74N4MNKv9jwYSMgFgJYCYiVIBYGyMEABjm5zbCDGA4Tw5n87iaKiTnItESwyfjBOFhcBoCzvYUTB78VueYAQlvoCDVZV5ws65ibRXBvH5ufOsKcO9Zh3flr6DwERQsn/d7U9KGBXl1yot0hwocHELcV1+JcxDH2g627owFPZyNHN3wBYwyv8CARS4xbj63CZqwAZMgg12e+QIGtg12By9gZXIMZ9SE5jIUecXFAbPRaXAyIg0Gcie/Jr5hM4oO1Mac2Uxi9YgsgudOwRyJkWSPkWsM0OGyU26JU2KJkWQ3R7FxiLjeujlOB3mFxk+copz048rxasQie2nz6X27lQMdSlgIc36SBfobSQK8uKdGOIOG6QSxZNpyLchBb6iBbsP9tjFjoXnQlR9hOHVu5pnUR+aHEXPgSWxPrM17EInE2Dn2EHdEl9Fhd9IqLHnHRI256LS6GxDHiunYTI8uEyTZhKuMDJx9nEsZjorhMFCexk98tJAL8qR1zIIqFMBZCYiWMFb/YaSnIZTBmZyhuZyDmoDmSwaGQ/eTrZltizLVH+Xp+OXntrUSGdfxLXHNp8u8f8zOwVXgoq8lly+tDzC+cjeP4O3D5F8/lx68ukkkFehG5BfgBieMAf2qM+ftR+Z8G/jL5dAj4mjFmVzLvGDAIxICoMRdrDbi61EVa/ESO+bDk2BM3HK3j3Cg1hvx9m+ifs5ywJ5O3eIRcU0ZR5wp2Wj0sc+zhRuczNJhivh76FgdtlSN+k7JMiLx4kJpYL14TJM8EyYsH8JogLmLndHtWADtx7MTJODHH3kBuitGiMimhOWKjKWqjKWLjg7Cd17Kq+XTzH/jTHeUUdTayqDSbAkc1+6Nv4I/247GdmisvIlz5sbn87h+3s6vgc1x+/IHEOH2qTxRqWpsw0IuIFfgRcBPQBGwVkaeNMcO7AEeB64wxvSJyK/AAcMWw/OuNMV0odREYY4g2+Yk0+rHmOXDMz0YsI4NVKBanPxKlPxLD3XSItf2dPLHkVjZ2PU24oJVA02f4hXMRf2F7jC/anmZTbCl/F/wcmUbYYI6SFw+SZwLkmhAO4uO05MLKtBgWOCMscCa2PTAGIrO9WA4argvU8/v2XHY29uHItOGshE2Nu7i6/CrcDuvJa5RU51C9qpD39y5jqXcId+8xyJtzkd6ROluT6dGvAeqMMfUAIvIocAdwMtAbY94ZVn4zUDGVjVTqbMXicYLHfEhbEL/XRnOxhf7uAQaiMfojUQYiie+hYScsfX3X64QsNl4tmUfU+33c4WrWZazji77/jzW29zkQWkKd/2o+wdHTvPL0IwKu0sTirj+K7mfxrZ/meLePnS0uDgGbGnbz5u48FpdmUzvLy9yixLm4a++o5uiuTrYO3c21x9/RQD8DTSbQlwPDN6puYmRvfbQvAs8Pe26Al0TEAP9hjHkgVSURuQ+4D6CqqipVEaUAiMbiDIWip76CUXzJx4PJ775QlMGhCFcNWVkZtrHdEeXVeACSGzh6rBay7Va8DhuzPE5y7DZy7Fa8EueW53fSt2gtyxcc4m0GuN36Le4yP2ex8312B1azI7iW6TQ//kzEnS5C+YXYjrVitQjVhZlUF9bwo7oicqv6yM7NY2djH3ua+8lx22kfCPJHtZUsWlfKvrdvYcXe58lZ9emL/TbUGZpMoE/1PzrlHC4RuZ5EoL96WPJVxpgWESkC/iAiB40xb465YOIPwAMAtbW1OmH3EhSOxun1hxkMRhkKRRgMRpOPowwEIwwlnweG7TkznMNmIdNpI9Npo8Dj4KYeK6VhQ2eelfxSD5+zWcm0Wcm2W7FbUt+Ezd/3NvaQn6bla9nKA8zlMm7veY3FA4+zL7hiRgf5EwKlVeQc2olEIxhb4oZthWcRDf69fGp5KbcuLeFA2yDbj/fww9fq+OFrddxcXchKi+G9nUVs+HQcxvn5qelpMoG+iRNzsRIqgJbRhURkOfBT4FZjTPeJdGNMS/J7h4g8SWIoaEygV+nPGMNgKErnYIheX5gef5geXzj5OIIvNHYDL7tVyHLZyXTaKMxyUl2YQabTTpbTRqbLdjKwZzhtOJIzaEzM4HuzncjQEPZKD1UVnpOrXSdStOs1gjmFvDTrMGEC3OOvYHn/v3Mw60629pYx04M8QKC8ity92/HUH8I3fykAFe7F7B94g/5IO7mOEpaV57CsPIf1Cwp5ZEsDD7/XQJmjiZjvCgKPPc9Nd92Mx6GT9maKyfxLbQVqRGQO0AzcA3xqeAERqQKeAD5jjDk8LD0DsBhjBpOPNwDfm6rGq+krEovT1h+kuS9A+0Aw+RUa0Ru3COS47eRlOFhcmoXX48DrcZDltpHttJPpsuG0WSYdpAFMNI7vtTYiTX7sszOwl3kmXdfZ10HOsT3UXfNhdspLrIqv4uMd/0mbayWbC74FDb89o5/BdBUsrcQIZB7YfTLQV7qXANAY2E+u49QmbWW5bv7HhgV84/p5vLBpL92P1dPwdpirD73KZ6+azZ9cORtvhiPl66jpY8JAb4yJisg3gRdJTK980BizT0S+msy/H/gOkA/8OPlLeWIaZTHwZDLNBjxsjHnhvLwTddEYY+jxhTnW7aOxJ0BTn5/2/hCx5JJ5l91CcZaLpeU5FGc7KcpykZfhIMdtx2qZuh6yicQZeqWVaGsAz5WFZ9z5Ltz9OgbhmWW9iBH+Z/sOIhYPrxf9L4zY0qAvnxB3uYkX55F1cBftdyT6bAXOKpyWDJoC+1iWc8OYOi67lTvXr+Dg5gd55dgdbHC5+P7LH/Afb9Rzz5pKvnRNNeW57gv9VtQkTeqzlzFmI7BxVNr9wx5/CfhSinr1wIpzbKOaho51+XjnSDePbm3gWJePgWBi2MVps1DhdXN1TQHluW4qvG5y3PYz6pWfDROOMfiHVmKdQTzXFOGcl03o0OT3o5dYlKKdr9A5Zz7v5mzlLl8ONcG9vFj6fQK2gvPY8osjWl1GxtZ9WEJB4k4XFrFS7l6UcuHUcAuuKONQyy4crat45ptreWhHE7969zi/evc4t68s46vXzWV+8cU7YEWlpoNsalKCkRhbjvbw6sEOXj/UwbFuPwBZThuzCzKYk/wqzHJiucALauLBGEMvtRDrCZGxvgTH7MwzvkbegXdxDPXy7G0FeIyN/961hx1599HmXn0eWnzxRRdU4XxnL1l7d9B/WWLTtkr3Yt7wbRuzcGo4WbCB67Lv5dHeH9Hyeiv//KUV/PcN8/nZW0d5ZEsDT+xo5sZFRXxt/Vwum5V3Id+SOg0N9Gpcrf0BXj3YwWsHO9lU10UgEsNps7Bubj5fuHoOV88r4N0j3ee9t346scEIQy+1EPdFyfxQKfbKjDO/iDGUbt3IQF4BL1bX82e9g/S71rEnJ32nEcZmlRD1ZJCzc/PJQD8rYwV0wVHf+yzJWZ+6YulKcgtdXObezJZtVzJ3VQfzLiviOx9dzJ/eMI9fvnuch945ysd/8i5r5uTx9fVzuW5+4UX9P6I00KtR2vqDPLenlWd3t/B+Qx8AFV43d9dWcP2CItZW549YObm5vudiNZVod4ihP7RAzJB1cxm24rMbI85sPkxm6xF+u6GQghjc6XfzYtlfw2Q2G5uprBYGl11O9q4tif3pLRbKXPPxWHOo820ZP9CLwLK7Wf3mv3Ks8npe//VBSqpzyPQ68WY4+PMba/jytXN4ZEsjP32rns/9fCtLyrL52vq53Lq0dErvyajJ00Cv6BwM8fzeVp7d1crW4z0YA4tLs/mLmxdw85Ji5hZmTrseWaTVz9ArbYjdQtaHy7Hmnv3Mj9ItzxFyOXl6WQ/f7hvgnaJ/IWzNnsLWTk/9q67A+97reOoP4Z+3CBEL1Rm11A29R9zEsIg1dcVln8D65j9y0+X7+c2z83jlF/u5/c9WntxWwuOw8cWr5/CZtbN46v1m7n/jCN98+H3mFBzmK9dW87HV5Tht41xbnRca6C9RPb4wL+xt49ndLWyu7yZuYH5xJv/txvl8eHkpcwvPfJz7Qgl9MID/nQ4s2Q6ybirFkmmfuNI4XF3N5B16j5fX2KkgQpHzSxxxLpjC1k5fA8vXELfZ8b73Ov55iYNS5mVezt6BV2gOHKTSsyR1xcIFULKc3OMPc/Xdv+L1Xx9i58uNrNowckW7w2bhjy6v5OOXVfDivjZ+/Hod335iD99/+QO+dM0c7l1TRYZTQ9CFoD/lS0i/P8KL+9t4dncrm+q6iMUN1QUZfPP6eXxkRdm0ny1h4obAti5C+/qxlbnJWF+CxXluPcOKtx8nZhV+c0WMPwvM5UjOnVPU2ukvlpFJ/6q1eN99jZZPfgljszMnYzUWrNQNvTd+oAdY/kl46a9Z/JFuGlcV8u6TdeRXZFC1OH9MUatFuG1ZYsXt23Vd/Pi1I/zv5w7ww9fq+JMrZ/O5dToX/3zTQJ/mBoMRXj7QzrO7Wnnzg04iMUNlnpv7rq3mI8tLWVyaPe2GZVKJh2L43mgn2uzHuSgH95qCMTtQnil3ZxP5+9/h+StgkcVKOPt/XXJb8PZcswHv1rfI3rWV/svW4bJmMDtjFfsG3uC6wj8Zv+KqT8Nr/wfZ/BNu+JN/p68jwEs/3ccn/rKW3OLUi9REhGtqCrmmppAdDb385PUj/OCVD/iPN4/wsVUVfP6q2dO+szFTaaBPQ75QlFcOdvDc7hZeO9RJOBqnLMfF59bN5iPLy1hekTMjgvsJ0Z4QvtfaiA9F8KwrxLkg9dS/M2IMs155iIjD8NQVFj5t+29ELZfegp+BpZcRyckj/80XT86+WZFzE0+2/B3HfDuBcXaqdHsTwX7bz3Hc+F1u+9oyfvv323jux7v5+P97Ga6M0w+nra7y8p+freVw+yA/33SUJ3Y08ciWBq6eV8Dnr5rN9QuKsOiN2ymjgT5NDIWivHKgnY17Wnn9UCehaJyiLCefWlPFR1eUsqrSO+N+cYwxhA4OENjahTgtZN5cjr1kaoJx7gfbya3fzS9uEJY7VuB0XKLn4VitdF97M8XPPoqztZFQaSXzMq/Abc1mV/9LwMfGr3vFV2HLf8KWB8j+0He49SvL+P0P3ufpH+zk9j9fOWGwB5hfnMXf3bWcv7h5IY9saeBX7x7ni7/YRmWem0/WVnJ3bSXF2ZM/g1elpoF+BhsvuN9zeSW3LSuldnbejJ3OFvdH8b/TQaTRj63cQ8a1xVhcUzNTwxIOUvPSv9OeZ9iyOoNP2r81JdedqTo33EnRi09Q/OxvaPjy/4PNYmdJ9vXs6H2OrkAXBe5xVgbnz4XFt8Pm++HyL1NWU8qtX1nG8/+xh2f+LRHsnZ7J3SjPy3Dwjevncd+11bywt41HtjTwf186zL++/AHXLyji3jWVXDe/EJs1jae8nkca6GeYxh4/rx7s4JWDHWw+0k04lgju966pSgT3WTOv5z6cMYZw3SCBLV2YmMG9pgDn4qkdalr6/P/BMuDnh39s4xr7t3DIpTdkM1w0O5eu9bdS+MoztN35x4QLS7jM+2G29z7Dz/b8jL9c85fjV77xb+DQC/Dyd+GuB5i9rCAR7O/fw1P/+j4f/vpyMr2T75HbrRY+uqKMj64o41iXj0e3NvL49kZePtBOQaaDDy8r5faV5ayuyp1Rw48Xmxgz/bZ+r62tNdu2bbvYzZgWorE47zf28cqBDl492M7h9iEAqgsyuH5hETcvKbmowf3h9xqm7FrRriD+97qIdQSxFbvwXFWENefsZ2Ok2uumZufPyd/4PE+tFQ6sv4nrLWO2aBqXbH/krNtyvvVUjb1vMd9aPG75wvJujlTdffK5vaeLRd/+IkOLVlD/rb8BEZ5r/T4HB9/gubueoySjZNxr8crfwlv/Fz7/Asy6EoDje7t58ad7sTms3HrfUkrn5Z71e4vE4rx6sIOnd7bw8oF2QtE4FV43H11RxobFxayoyJ3RnZupIiLbxzuTWwP9NGOMoa5jiHeOdPPukW7ere+mPxDBZhHWzMnjhoVF3LCwiOppMs99KgJ9bCBMcFcv4bpBxGXFfVkejppznw00OtDXHHmEvMefoK5E+Mm9c7nL/j1sMvk5+Okc6AEKX/gdFY88wNFv/DV9a66lL9zOfx67jw2zNvAP1/7D+C8e9sGP1yZW2N73OmQWJtrU4mPjT3Yz2BNk7R1zWfGhCiznOPQyGIzwh/3t/H5nC28npwgXZDq5YWEhNy4q5uqagkt2n/zTBfpL8ycyjYSiMfa3DPB+Qx/bG3p5r76HrqEQAOW5bjYsLub6hUVcXVNAtuvsFwZNR9GeEKG9fYTrB8EiOJfk4l7pRRxTvGrSxFlR92PcT79OV4aFH99VwG32vzijIH8p6LzpTvLefY3Kn/+AQMVscsuquG/Zffx414+5tuJaPlz94dQVHRnwyf+Cn22A334OPvMk2BzklWXwiW/X8uovD/DOE3Uc3trG9X+8kKJZZ7/qOMtl567VFdy1uoI+f5g3Dnfyh/3tPL+njce2NeGwWlhZmcvaufmsrc5jdZUXl11X4WqP/gIKRmJ80D7EgdYB9rcOsLupj70tA4SjcQDKclxcUZ3PldX5XDk3n8q8yR+acbGcaY/eROKEG3yED/UTbQ+CVXAuzMa11IvFM7X9jtChfjzRTq7Y8R38b3TS7bbwT/fmsj7vb/BK6RlfL9179ACOzjbmf+/PiTvdfPBX/8jHblnJF178Aod7D/PATQ+wvHD5+I3Y/Rg88WWoXg9/9CtwJQK6MYYjOzp56zeH8Q+GmbuqiMtunUVh5dTNmQ9H42w91sObhzvZXN/NnuZ+4iaxOnd5eQ7LK3JZUZn4Pjt/8ieOzSQ6dHOB9fsj1HcNcbTLx9EuH/WdPg63D1Lf5SMWT/y8PQ4rS8qyWV3lZVVVLquqvDNyGtlkAn3cFyXS4ifS4CPS7IeYwZJlx7kwG0dN9jmvbk3FHhlg7qb7Kdj0HL7DLo4XwgN3V3Bdzl+RLYVndc1LIdADeI4cZN4//RVxh5N5P/wBA4sr+fwLn6cn2MM/r/9nri6/OmU9AHY+DE//KeTXwEd/AFVXnMwK+SO8/1IDe15vIhyMUb7Ay/w1xcxdXYTTPbV/5AeCEbYd6+HdI93saOhjX0s/wUiiQ5XtsrGwJJt5xZnUFGVSU5TFvKJMirOdM/oPwDkHehG5BfgBiROmfmqM+ftR+ZLMvw3wA58zxuyYTN1UpmOgN8YQiMTo80fo9YfpGgrT1h+gtT9Ia1+Q1oEgrX0B2vqDDA47+9RqESq9buYWZrKoNJvFZdksKs1mVp4nLW4gDQ/0xhhMKE6sJ5T46g4R7Q4R748AIG4rjtmZOOZkYi1yTfkvlSUWorh7K+V7fo/ZtpVwnQNLRHhxtbDz+htYZ/8MTjn7T0mXSqAHcDUfp/pfv4uzs5Xs227FfPw2/rTzh9T1H+G2Obfx2SWfZUn+OFskHHkNnvo6DLbAotvhss/BnGvBmhgqC/kj7HmjmQPvtDLQGcBiE0qrcyhf4KV0bg75FZm4M6d2S4RoLM4HHUPsbupjd1M/h9sHOdw+RH8gcrKM02ah3Oum0uuhwuum3OumINNJYaaT/EwHBcnv03VDtnMK9CJiBQ4DN5E4KHwrcK8xZv+wMrcBf0oi0F8B/MAYc8Vk6qZytoH+QOsA0ZghGo8TN4ZozBAzhlj81Fc0nkgPRmL4IzGC4Rj+cIxAJJZIC0cJROIEwjEGghH6k4G9LxA5OcQy8ucDBZlOynJclOS4KM1xU5brYk5BJnMKMqjK85w8tHomCQeitB7pJx43mOTXicfhYIxwIErIH2Xv0V7i/ijxoShxXwQip/4/iceKLd+JrcSNrcyD1euYkuBec/wRMgKtmMZerPsbORTvQSIRsnujZPcI7qAQtcCWBcLmdYuYXfQpymT+Ob/upRToASzBADfsfIGeX/wS4/dj8XrpmJXF6zltPLYuTpG7iGWFy/jeVd8j2zFq3D00BG/9M2z/OQR6we6BslWQPw82/C24cjDG0H5sgCPbO2g61EvX/9/e/YbIUd9xHH9/dnZ29/7mNDlJGkMMVWy1iD5Jg1gRGloRMfVBoQ9KRQkhD0LTBxalakMVH5QWERRpFAUF21JIBal/I1ZsA6maENuYxDSVhrsk5Lw/Jpm78/6Yrw9mLm7O27vZy60zt35fcNzN3sze5/Zuv7vzm9+f3giSf5/WzhKdyyq0LSnT2lWmbUmJcmtIWA4IywGrr15KEF7Y88rM6I/G+W/fGf7XF9EzNErP4Ai9Q6P0Do0wNDIx43EdlSJL20p0toS0leJF6juSBeqnFqwvFwuUigXCoEApKBAW48+los7dVgwKFAQFiYKEFDc1zXcaiAu9GLsWOJIsC4ikPwMbgOpivQF4zuJXjd2SuiStAC5LceyCuf2JXedOz+pVCgpUwgKtpSItpYBKGNBRLrJ6aSvXruqiqzWkq7XERcnnpe0lViypcElHZVEW8rmcHhjlb4+/P+s+hUBYWKDQWiToDAlXtFBoLxJcVCK4uExhgU/Hp1x68k26h/Zx6nCR/v0drAzERAAfdxX5/7cqHL98NdE161g+upbr9eVJtlw6ZystXLJ1K8s2buT0zp2MvPMu4YED/LTru1y5bj17Tu7ho1Mf0R7O0AOs3A7rt8FN98Lh1+DoLji2Fw6/Crf8Hojnvlm+ZgnL18QvUp8OT9B39DQDx4YZPB5xZnCMwRPD9BwaYnx08ry73/jI9y640Euiu6NMd0eZ67/55UFhI+OTDETjfByNMRCN0x+NMRCN0R+NMzA8zplPJxgem6RncIRobJLhsUmisUkmPpt/c/iy9jLv3b/+Qn6tGaV5Jq4Eeqq2e4nftc+1z8qUxwIgaROwKdmMJH2YIttXaRnQn3WIlL7mWfcCLyzsXcaa7HG9e847qbnG1vanzn0ZUGdTxi/L02+p+3Hdsr2+H7nAGvZ/cBTQA/M+fHWtb6Qp9DOda09/yaq1T5pj4xvNngSeTJEnE5Leq3ValDeetTE8a2Mspqyw+PJCukLfC6yq2r4UOJ5yn1KKY51zzjVQmkaud4ErJK2RVAJ+Arw4bZ8XgZ8ptg44ZWYnUh7rnHOugeZ8R29mk5K2AK8Rd5F8xsw+kLQ5+f4fgJeJe9wcIe5eeedsxzbkN2m83DYrzcCzNoZnbYzFlBUWX958Dphyzjm3cJqvX6BzzrnzeKF3zrkm54U+JUkPSfq3pH2SXpf0jawzzUbS7yQdSjK/fC4d/AAAAplJREFUIGn+E4I3mKQfS/pA0llJuey2JulmSR9KOiLp3qzz1CLpGUl9kvZnnWUuklZJ+rukg8nff2vWmWqRVJH0jqT3k6y/yTpTPbyNPiVJnWZ2Ovn658BVZrY541g1SfoB8GZyQfy3AGY2y1JB2ZH0beAssB2428xyNdHRfKfyyIKkG4GIeKT6d7LOM5tk9PwKM9srqQPYA/wop4+rgDYziySFwD+BrWa2O+Noqfg7+pSminyijRoDv/LCzF43s6lx47uJxzDkkpkdNLO8jYSudm4aEDMbB6am8sgdM3sbGMw6RxpmdmJq8kMzOwMcJB5NnzsWi5LNMPnIdQ2o5oW+DpIeltRDPDL811nnqcNdwCtZh1jEak3x4RaIpMuA64B/ZZukNkmBpH1AH7DTzHKbdTov9FUkvSFp/wwfGwDM7D4zWwU8D2zJNu3ceZN97gMmiTNnJk3WHEs9lYern6R2YAfwi2lnzrliZp+Z2bXEZ8drJeW6aayaLyVYxczSThv3R+AlYFsD48xprryS7gBuBb5vGV+MqeOxzaM004C4eUjau3cAz5vZX7POk4aZfSLpLeBmIPcXvcHf0acm6YqqzduAQ1llSSNZ8OUe4DYzG8k6zyLnU3k0QHKB82ngoJk9knWe2Ujqnuq5JqkFWE/Oa0A173WTkqQdwJXEvUOOApvN7Fi2qWqTdAQoAwPJTbvz2ktI0u3AY0A38Amwz8x+mG2q8yWL6zzKF1N5PJxxpBlJ+hNwE/FUuieBbWb2dKahapB0A/AP4D/EzyuAX5nZy9mlmpmka4Bnif/+BeAvZvZgtqnS80LvnHNNzptunHOuyXmhd865JueF3jnnmpwXeueca3Je6J1zrsl5oXfOuSbnhd4555rc51b93cptv29OAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "SEQ_LEN = 10\n",
    "def create_time_series():\n",
    "  freq = (np.random.random() * 0.5) + 0.1  # 0.1 to 0.6\n",
    "  ampl = np.random.random() + 0.5  # 0.5 to 1.5\n",
    "  x = np.sin(np.arange(0, SEQ_LEN) * freq) * ampl\n",
    "  return x\n",
    "\n",
    "for i in range(0, 5):\n",
    "  sns.distplot( create_time_series() );  # 5 series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_csv(filename, N):\n",
    "  with open(filename, 'w') as ofp:\n",
    "    for lineno in range(0, N):\n",
    "      seq = create_time_series()\n",
    "      line = \",\".join(map(str, seq))\n",
    "      ofp.write(line + '\\n')\n",
    "\n",
    "to_csv('train.csv', 1000)  # 1000 sequences\n",
    "to_csv('valid.csv',  50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> train.csv <==\n",
      "0.0,0.22299305220216323,0.4154388426580984,0.5509747027780788,0.6110339133114452,0.5873891128838562,0.48327934370073466,0.31296634323851563,0.09978086441764555,-0.12707334601853693\n",
      "0.0,0.15901002275298337,0.312416176599827,0.4548120855104597,0.5811793991162043,0.6870646506027852,0.7687362068432376,0.8233157794969398,0.8488798623294476,0.8445275198750664\n",
      "0.0,0.1393968329251676,0.2703258427596177,0.38483359419599195,0.47596418040820354,0.5381817677190796,0.5677068762693995,0.5627459689271358,0.5236004017910049,0.452648117963003\n",
      "0.0,0.224754084492425,0.4198144336661288,0.5594103462116253,0.625098891100264,0.6082015240920136,0.5109506671010717,0.3461947683037169,0.13570080943082635,-0.09272147286664245\n",
      "0.0,0.141784381770469,0.28206183491143016,0.4193414469008347,0.5521641672073662,0.6791183145345397,0.7988545806098002,0.910100371053667,1.0116733309097004,1.1024939110817333\n",
      "\n",
      "==> valid.csv <==\n",
      "0.0,0.42040678054096164,0.7119890871863938,0.7853978918294988,0.6181386699402988,0.2614643562034482,-0.1753299976692934,-0.5583983039297977,-0.7703576484477038,-0.7462577204558432\n",
      "0.0,0.616679077395263,1.0497745893976878,1.1703552633834748,0.9425245838728331,0.4341070806472849,-0.20354283624165745,-0.780598643557034,-1.1252725080143726,-1.1349560239546488\n",
      "0.0,0.4207947754084429,0.761315754420054,0.9566027394860507,0.9694014000056725,0.7972701727068776,0.47304603157611436,0.05858027376127529,-0.3670606726338375,-0.7226785212571388\n",
      "0.0,0.3867903867516891,0.7270290515972765,0.9797669527557336,1.1145861044320475,1.1152604992811048,0.9817089712694579,0.7300049643219796,0.39044203105892356,0.003887886346817335\n",
      "0.0,0.8380791909012454,1.3850202332606303,1.4508227021030047,1.0126276490661605,0.22265851296875946,-0.6446593843514298,-1.2880307362402985,-1.4839564435108439,-1.1643743847242083\n"
     ]
    }
   ],
   "source": [
    "!head -5 train.csv valid.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> RNN </h2>\n",
    "\n",
    "For more info, see:\n",
    "<ol>\n",
    "<li> http://colah.github.io/posts/2015-08-Understanding-LSTMs/ for the theory\n",
    "<li> https://www.tensorflow.org/tutorials/recurrent for explanations\n",
    "<li> https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb for sample code\n",
    "</ol>\n",
    "\n",
    "Here, we are trying to predict from 9 values of a timeseries, the tenth value.\n",
    "\n",
    "<p>\n",
    "\n",
    "<h3> Imports </h3>\n",
    "\n",
    "Several tensorflow packages and shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "import tensorflow.contrib.metrics as metrics\n",
    "import tensorflow.contrib.rnn as rnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Input Fn to read CSV </h3>\n",
    "\n",
    "Our CSV file structure is quite simple -- a bunch of floating point numbers (note the type of DEFAULTS). We ask for the data to be read BATCH_SIZE sequences at a time.  The Estimator API in tf.contrib.learn wants the features returned as a dict. We'll just call this timeseries column 'rawdata'.\n",
    "<p>\n",
    "Our CSV file sequences consist of 10 numbers. We'll assume that 9 of them are inputs and we need to predict the last one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEFAULTS = [[0.0] for x in range(0, SEQ_LEN)]\n",
    "BATCH_SIZE = 20\n",
    "TIMESERIES_COL = 'rawdata'\n",
    "# In each sequence, column index 0 to N_INPUTS - 1 are features, and column index N_INPUTS to SEQ_LEN are labels\n",
    "N_OUTPUTS = 1\n",
    "N_INPUTS = SEQ_LEN - N_OUTPUTS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading data using the Estimator API in tf.estimator requires an input_fn. This input_fn needs to return a dict of features and the corresponding labels.\n",
    "<p>\n",
    "So, we read the CSV file.  The Tensor format here will be a scalar -- entire line.  We then decode the CSV. At this point, all_data will contain a list of scalar Tensors. There will be SEQ_LEN of these tensors.\n",
    "<p>\n",
    "We split this list of SEQ_LEN tensors into a list of N_INPUTS Tensors and a list of N_OUTPUTS Tensors. We stack them along the first dimension to then get a vector Tensor for each.  We then put the inputs into a dict and call it features.  The other is the ground truth, so labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read data and convert to needed format\n",
    "def read_dataset(filename, mode, batch_size = 512):\n",
    "  def _input_fn():\n",
    "    # Provide the ability to decode a CSV\n",
    "    def decode_csv(line):\n",
    "      # all_data is a list of scalar tensors\n",
    "      all_data = tf.decode_csv(line, record_defaults = DEFAULTS)\n",
    "      inputs = all_data[:len(all_data) - N_OUTPUTS]  # first N_INPUTS values\n",
    "      labels = all_data[len(all_data) - N_OUTPUTS:] # last N_OUTPUTS values\n",
    "\n",
    "      # Convert each list of rank R tensors to one rank R+1 tensor\n",
    "      inputs = tf.stack(inputs, axis = 0)\n",
    "      labels = tf.stack(labels, axis = 0)\n",
    "      \n",
    "      # Convert input R+1 tensor into a feature dictionary of one R+1 tensor\n",
    "      features = {TIMESERIES_COL: inputs}\n",
    "\n",
    "      return features, labels\n",
    "\n",
    "    # Create list of files that match pattern\n",
    "    file_list = tf.gfile.Glob(filename)\n",
    "\n",
    "    # Create dataset from file list\n",
    "    dataset = tf.data.TextLineDataset(file_list).map(decode_csv)\n",
    "\n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        num_epochs = None # indefinitely\n",
    "        dataset = dataset.shuffle(buffer_size = 10 * batch_size)\n",
    "    else:\n",
    "        num_epochs = 1 # end-of-input after this\n",
    "\n",
    "    dataset = dataset.repeat(num_epochs).batch(batch_size)\n",
    "\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    batch_features, batch_labels = iterator.get_next()\n",
    "    return batch_features, batch_labels\n",
    "  return _input_fn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Define RNN </h3>\n",
    "\n",
    "A recursive neural network consists of possibly stacked LSTM cells.\n",
    "<p>\n",
    "The RNN has one output per input, so it will have 8 output cells.  We use only the last output cell, but rather use it directly, we do a matrix multiplication of that cell by a set of weights to get the actual predictions. This allows for a degree of scaling between inputs and predictions if necessary (we don't really need it in this problem).\n",
    "<p>\n",
    "Finally, to supply a model function to the Estimator API, you need to return a EstimatorSpec. The rest of the function creates the necessary objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "LSTM_SIZE = 3  # number of hidden layers in each of the LSTM cells\n",
    "\n",
    "# Create the inference model\n",
    "def simple_rnn(features, labels, mode):\n",
    "  # 0. Reformat input shape to become a sequence\n",
    "  x = tf.split(features[TIMESERIES_COL], N_INPUTS, 1)\n",
    "    \n",
    "  # 1. Configure the RNN\n",
    "  lstm_cell = rnn.BasicLSTMCell(LSTM_SIZE, forget_bias = 1.0)\n",
    "  outputs, _ = rnn.static_rnn(lstm_cell, x, dtype = tf.float32)\n",
    "\n",
    "  # Slice to keep only the last cell of the RNN\n",
    "  outputs = outputs[-1]\n",
    "  \n",
    "  # Output is result of linear activation of last layer of RNN\n",
    "  weight = tf.get_variable(\"weight\", initializer=tf.initializers.random_normal, shape=[LSTM_SIZE, N_OUTPUTS])\n",
    "  bias = tf.get_variable(\"bias\", initializer=tf.initializers.random_normal, shape=[N_OUTPUTS])\n",
    "  predictions = tf.matmul(outputs, weight) + bias\n",
    "    \n",
    "  # 2. Loss function, training/eval ops\n",
    "  if mode == tf.estimator.ModeKeys.TRAIN or mode == tf.estimator.ModeKeys.EVAL:\n",
    "    loss = tf.losses.mean_squared_error(labels, predictions)\n",
    "    train_op = tf.contrib.layers.optimize_loss(\n",
    "      loss = loss,\n",
    "      global_step = tf.train.get_global_step(),\n",
    "      learning_rate = 0.01,\n",
    "      optimizer = \"SGD\")\n",
    "    eval_metric_ops = {\n",
    "      \"rmse\": tf.metrics.root_mean_squared_error(labels, predictions)\n",
    "    }\n",
    "  else:\n",
    "    loss = None\n",
    "    train_op = None\n",
    "    eval_metric_ops = None\n",
    "  \n",
    "  # 3. Create predictions\n",
    "  predictions_dict = {\"predicted\": predictions}\n",
    "  \n",
    "  # 4. Create export outputs\n",
    "  export_outputs = {\"predict_export_outputs\": tf.estimator.export.PredictOutput(outputs = predictions)}\n",
    "  \n",
    "  # 5. Return EstimatorSpec\n",
    "  return tf.estimator.EstimatorSpec(\n",
    "      mode = mode,\n",
    "      predictions = predictions_dict,\n",
    "      loss = loss,\n",
    "      train_op = train_op,\n",
    "      eval_metric_ops = eval_metric_ops,\n",
    "      export_outputs = export_outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Estimator </h3>\n",
    "\n",
    "Distributed training is launched off using an Estimator.  The key line here is that we use tf.estimator.Estimator rather than, say tf.estimator.DNNRegressor.  This allows us to provide a model_fn, which will be our RNN defined above.  Note also that we specify a serving_input_fn -- this is how we parse the input data provided to us at prediction time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create functions to read in respective datasets\n",
    "def get_train():\n",
    "  return read_dataset(filename = 'train.csv', mode = tf.estimator.ModeKeys.TRAIN, batch_size = 512)\n",
    "\n",
    "def get_valid():\n",
    "  return read_dataset(filename = 'valid.csv', mode = tf.estimator.ModeKeys.EVAL, batch_size = 512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create serving input function\n",
    "def serving_input_fn():\n",
    "  feature_placeholders = {\n",
    "      TIMESERIES_COL: tf.placeholder(tf.float32, [None, N_INPUTS])\n",
    "  }\n",
    "  \n",
    "  features = {\n",
    "    key: tf.expand_dims(tensor, -1)\n",
    "    for key, tensor in feature_placeholders.items()\n",
    "  }\n",
    "  features[TIMESERIES_COL] = tf.squeeze(features[TIMESERIES_COL], axis = [2])\n",
    "    \n",
    "  return tf.estimator.export.ServingInputReceiver(features, feature_placeholders)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create custom estimator's train and evaluate function\n",
    "def train_and_evaluate(output_dir):\n",
    "  estimator = tf.estimator.Estimator(model_fn = simple_rnn, \n",
    "                         model_dir = output_dir)\n",
    "  train_spec = tf.estimator.TrainSpec(input_fn = get_train(),\n",
    "                                    max_steps = 1000)\n",
    "  exporter = tf.estimator.LatestExporter('exporter', serving_input_fn)\n",
    "  eval_spec = tf.estimator.EvalSpec(input_fn = get_valid(),\n",
    "                                  steps = None,\n",
    "                                  exporters = exporter)\n",
    "  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'outputdir', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5b6e7b4a50>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into outputdir/model.ckpt.\n",
      "INFO:tensorflow:loss = 0.74787104, step = 1\n",
      "INFO:tensorflow:global_step/sec: 13.075\n",
      "INFO:tensorflow:loss = 0.2175352, step = 101 (7.652 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.8919\n",
      "INFO:tensorflow:loss = 0.11749355, step = 201 (5.025 sec)\n",
      "INFO:tensorflow:global_step/sec: 20.0455\n",
      "INFO:tensorflow:loss = 0.07706216, step = 301 (4.990 sec)\n",
      "INFO:tensorflow:global_step/sec: 19.931\n",
      "INFO:tensorflow:loss = 0.06313775, step = 401 (5.018 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.2121\n",
      "INFO:tensorflow:loss = 0.05669792, step = 501 (6.167 sec)\n",
      "INFO:tensorflow:global_step/sec: 14.5402\n",
      "INFO:tensorflow:loss = 0.046958122, step = 601 (6.880 sec)\n",
      "INFO:tensorflow:global_step/sec: 15.0092\n",
      "INFO:tensorflow:loss = 0.041990355, step = 701 (6.660 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.5967\n",
      "INFO:tensorflow:loss = 0.033966184, step = 801 (6.024 sec)\n",
      "INFO:tensorflow:global_step/sec: 18.3291\n",
      "INFO:tensorflow:loss = 0.027492512, step = 901 (5.456 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into outputdir/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2020-05-31T02:50:05Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from outputdir/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2020-05-31-02:50:06\n",
      "INFO:tensorflow:Saving dict for global step 1000: global_step = 1000, loss = 0.018401934, rmse = 0.13565373\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: outputdir/model.ckpt-1000\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['predict_export_outputs', 'serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from outputdir/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: outputdir/export/exporter/temp-b'1590893406'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.022302669.\n"
     ]
    }
   ],
   "source": [
    "# Run the model\n",
    "shutil.rmtree('outputdir', ignore_errors = True) # start fresh each time\n",
    "train_and_evaluate('outputdir')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3> Standalone Python module </h3>\n",
    "\n",
    "To train this on Cloud ML Engine, we take the code in this notebook and make a standalone Python module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process is terminated.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Run module as-is\n",
    "echo $PWD\n",
    "rm -rf outputdir\n",
    "export PYTHONPATH=${PYTHONPATH}:${PWD}/simplernn\n",
    "python -m trainer.task \\\n",
    "  --train_data_paths=\"${PWD}/train.csv*\" \\\n",
    "  --eval_data_paths=\"${PWD}/valid.csv*\"  \\\n",
    "  --output_dir=outputdir \\\n",
    "  --job-dir=./tmp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2> Variant: long sequence </h2>\n",
    "\n",
    "To create short sequences from a very long sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input= [ 1.  2.  3.  4.  5.  6.  7.  8.  9. 10.]\n",
      "output= [[1. 2. 3. 4. 5.]\n",
      " [2. 3. 4. 5. 6.]\n",
      " [3. 4. 5. 6. 7.]\n",
      " [4. 5. 6. 7. 8.]\n",
      " [5. 6. 7. 8. 9.]]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "def breakup(sess, x, lookback_len):\n",
    "  N = sess.run(tf.size(x))\n",
    "  windows = [tf.slice(x, [b], [lookback_len]) for b in range(0, N-lookback_len)]\n",
    "  windows = tf.stack(windows)\n",
    "  return windows\n",
    "\n",
    "x = tf.constant(np.arange(1,11, dtype=np.float32))\n",
    "with tf.Session() as sess:\n",
    "    print('input=', x.eval())\n",
    "    seqx = breakup(sess, x, 5)\n",
    "    print('output=', seqx.eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variant: Keras\n",
    "\n",
    "You can also invoke a Keras model from within the Estimator framework by creating an estimator from the compiled Keras model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_keras_estimator(output_dir):\n",
    "  from tensorflow import keras\n",
    "  model = keras.models.Sequential()\n",
    "  model.add(keras.layers.Dense(32, input_shape=(N_INPUTS,), name=TIMESERIES_INPUT_LAYER))\n",
    "  model.add(keras.layers.Activation('relu'))\n",
    "  model.add(keras.layers.Dense(1))\n",
    "  model.compile(loss = 'mean_squared_error',\n",
    "                optimizer = 'adam',\n",
    "                metrics = ['mae', 'mape']) # mean absolute [percentage] error\n",
    "  return keras.estimator.model_to_estimator(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/simplernn/trainer/model.py:21: The name tf.logging.set_verbosity is deprecated. Please use tf.compat.v1.logging.set_verbosity instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/simplernn/trainer/model.py:21: The name tf.logging.INFO is deprecated. Please use tf.compat.v1.logging.INFO instead.\n",
      "\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Using default config.\n",
      "INFO:tensorflow:Using the Keras model provided.\n",
      "2020-05-31 02:51:45.233162: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-05-31 02:51:45.239630: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
      "2020-05-31 02:51:45.240159: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x563bc8efb0d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2020-05-31 02:51:45.240199: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_session_creation_timeout_secs': 7200, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f63050529d0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Not using Distribute Coordinator.\n",
      "INFO:tensorflow:Running training and evaluation locally (non-distributed).\n",
      "INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps None or save_checkpoints_secs 600.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n",
      "WARNING:tensorflow:From /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/simplernn/trainer/model.py:53: The name tf.gfile.Glob is deprecated. Please use tf.io.gfile.glob instead.\n",
      "\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/autograph/converters/directives.py:119: The name tf.decode_csv is deprecated. Please use tf.io.decode_csv instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/simplernn/trainer/model.py:66: DatasetV1.make_one_shot_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_one_shot_iterator(dataset)`.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Warm-starting with WarmStartSettings: WarmStartSettings(ckpt_to_initialize_from='/home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/keras/keras_model.ckpt', vars_to_warm_start='.*', var_name_to_vocab_info={}, var_name_to_prev_var_name={})\n",
      "INFO:tensorflow:Warm-starting from: /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/keras/keras_model.ckpt\n",
      "INFO:tensorflow:Warm-starting variables only in TRAINABLE_VARIABLES.\n",
      "INFO:tensorflow:Warm-started 4 variables.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/ops/array_ops.py:1475: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/model.ckpt.\n",
      "INFO:tensorflow:loss = 1.4711411, step = 1\n",
      "INFO:tensorflow:global_step/sec: 15.0111\n",
      "INFO:tensorflow:loss = 0.054738425, step = 101 (6.662 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.8024\n",
      "INFO:tensorflow:loss = 0.010876194, step = 201 (5.952 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.3931\n",
      "INFO:tensorflow:loss = 0.005013787, step = 301 (6.100 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.015\n",
      "INFO:tensorflow:loss = 0.0032372954, step = 401 (6.244 sec)\n",
      "INFO:tensorflow:global_step/sec: 16.2979\n",
      "INFO:tensorflow:loss = 0.0024657317, step = 501 (6.136 sec)\n",
      "INFO:tensorflow:global_step/sec: 15.7945\n",
      "INFO:tensorflow:loss = 0.0014497541, step = 601 (6.332 sec)\n",
      "INFO:tensorflow:global_step/sec: 9.77963\n",
      "INFO:tensorflow:loss = 0.0010798911, step = 701 (10.225 sec)\n",
      "INFO:tensorflow:global_step/sec: 14.3505\n",
      "INFO:tensorflow:loss = 0.0007206198, step = 801 (6.969 sec)\n",
      "INFO:tensorflow:global_step/sec: 14.6158\n",
      "INFO:tensorflow:loss = 0.00046781416, step = 901 (6.842 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/model.ckpt.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2020-05-31T02:52:55Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/model.ckpt-1000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2020-05-31-02:52:56\n",
      "INFO:tensorflow:Saving dict for global step 1000: global_step = 1000, loss = 0.00063822814, mean_absolute_error = 0.018170318, mean_absolute_percentage_error = 13.573708\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 1000: /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/model.ckpt-1000\n",
      "WARNING:tensorflow:From /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/simplernn/trainer/model.py:138: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/model.ckpt-1000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /home/jupyter/training-data-analyst/courses/machine_learning/deepdive/05_artandscience/outputdir/export/exporter/temp-b'1590893576'/saved_model.pb\n",
      "INFO:tensorflow:Loss for final step: 0.0003437588.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "# Run module as-is\n",
    "echo $PWD\n",
    "rm -rf outputdir\n",
    "export PYTHONPATH=${PYTHONPATH}:${PWD}/simplernn\n",
    "python -m trainer.task \\\n",
    "  --train_data_paths=\"${PWD}/train.csv*\" \\\n",
    "  --eval_data_paths=\"${PWD}/valid.csv*\"  \\\n",
    "  --output_dir=${PWD}/outputdir \\\n",
    "  --job-dir=./tmp --keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Challenge Excercise\n",
    "Given the trainer code provided in the simplernn folder, make a call to AI Platform using the gcloud command to train the model in the Cloud."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "gcloud ai-platform \"TODO: Insert code here with all the neeed parameters\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright 2017 Google Inc. Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
